{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import math\n",
    "from tqdm import tqdm\n",
    "from sklearn.metrics import mean_squared_error,explained_variance_score\n",
    "from sklearn.model_selection import KFold\n",
    "import lightgbm as lgb\n",
    "test_data_path = '../data/A_testData0531.csv'\n",
    "train_gps_path = '../data/complete_train.csv'\n",
    "port_path = '../data/port.csv'\n",
    "result_path = '../result/result_local_v2.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['CNYTN-MXZLO', 'CNSHK-MYTPP', 'CNSHK-SGSIN', 'CNSHK-CLVAP',\n",
       "       'CNYTN-ARENA', 'CNYTN-MATNG', 'CNSHK-GRPIR', 'CNSHK-PKQCT',\n",
       "       'COBUN-HKHKG', 'CNYTN-PAONX', 'CNSHK-SIKOP', 'CNYTN-CAVAN',\n",
       "       'CNSHK-ESALG', 'CNYTN-MTMLA', 'CNSHK-ZADUR', 'CNSHK-LBBEY',\n",
       "       'CNSHA-SGSIN', 'CNYTN-RTM', 'CNHKG-MXZLO', 'HKHKG-FRFOS',\n",
       "       'CNYTN-NZAKL', 'CNSHA-PAMIT'], dtype=object)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def format_data_type(data, mode='train'):\n",
    "    if mode=='test':\n",
    "        data['onboardDate'] = pd.to_datetime(data['onboardDate'], infer_datetime_format=True)\n",
    "        data['temp_timestamp'] = data['timestamp']\n",
    "        data['ETA'] = None\n",
    "        data['creatDate'] = None\n",
    "    data['loadingOrder'] = data['loadingOrder'].astype(str)\n",
    "    data['timestamp'] = pd.to_datetime(data['timestamp'], infer_datetime_format=True)\n",
    "    data['longitude'] = data['longitude'].astype(float)\n",
    "    data['latitude'] = data['latitude'].astype(float)\n",
    "    data['speed'] = data['speed'].astype(float)\n",
    "    data['TRANSPORT_TRACE'] = data['TRANSPORT_TRACE'].astype(str)\n",
    "    return data\n",
    "\n",
    "def get_test_data_info(path):\n",
    "    data = pd.read_csv(path) \n",
    "    test_trace_set = data['TRANSPORT_TRACE'].unique()\n",
    "    test_order_belong_to_trace = {}\n",
    "    for item in test_trace_set:\n",
    "        orders = data[data['TRANSPORT_TRACE'] == item]['loadingOrder'].unique()\n",
    "        test_order_belong_to_trace[item] = orders\n",
    "    return format_data_type(data, mode='test'), test_trace_set, test_order_belong_to_trace\n",
    "\n",
    "test_data_origin, test_trace_set, test_order_belong_to_trace = get_test_data_info(test_data_path)\n",
    "test_trace_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_port_info():\n",
    "    port_data = {}\n",
    "    test_port_set = set()\n",
    "    for route in test_trace_set:\n",
    "        ports = route.split('-')\n",
    "        test_port_set = set.union(test_port_set, set(ports))\n",
    "    port_data_origin = pd.read_csv(port_path)\n",
    "    for item in port_data_origin.itertuples():\n",
    "        if getattr(item, 'TRANS_NODE_NAME') in test_port_set:\n",
    "            port_data[getattr(item, 'TRANS_NODE_NAME')] = {'LONGITUDE': getattr(item, 'LONGITUDE'),'LATITUDE': getattr(item, 'LATITUDE') }\n",
    "    return port_data\n",
    "port_data = get_port_info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_train_order_by_route(route):\n",
    "    ports = route.split(\"-\")\n",
    "    start_port = ports[0]\n",
    "    dest_port = ports[-1]\n",
    "    train_order_by_route = None\n",
    "    train_data_origin_chunk = pd.read_csv(train_gps_path, chunksize = 2000000, header=None)\n",
    "    for chunk in tqdm(train_data_origin_chunk):\n",
    "        valid_order_name = chunk[chunk[8].apply(lambda x: (start_port in str(x)) and (dest_port in str(x)))][0].unique()\n",
    "        if (valid_order_name.size > 0):\n",
    "            valid_order_info = chunk[chunk[0].isin(valid_order_name)]\n",
    "            train_order_by_route = pd.concat([train_order_by_route,valid_order_info])\n",
    "#             break\n",
    "    train_order_by_route.columns = ['loadingOrder', 'carrierName','timestamp','longitude','latitude','vesselMMSI','speed','direction','TRANSPORT_TRACE']\n",
    "    train_order_by_route['timestamp'] = pd.to_datetime(train_order_by_route['timestamp'], infer_datetime_format=True)\n",
    "#     data = format_data_type(train_order_by_route, mode='train')\n",
    "    return train_order_by_route\n",
    "\n",
    "def get_train_data(route_order_info, route):\n",
    "    ports = route.split(\"-\")\n",
    "    start_port = ports[0]\n",
    "    dest_port = ports[-1]\n",
    "    start_longitude = port_data[start_port]['LONGITUDE']\n",
    "    start_latitude = port_data[start_port]['LATITUDE']\n",
    "    dest_longitude = port_data[dest_port]['LONGITUDE']\n",
    "    dest_latitude = port_data[dest_port]['LATITUDE']\n",
    "    train_data = None\n",
    "    order_list = route_order_info['loadingOrder'].unique()\n",
    "    print(route, order_list.shape)\n",
    "    for order in tqdm(order_list):\n",
    "        order_info_set = route_order_info[route_order_info['loadingOrder'] == order].sort_values(by='timestamp').reset_index(drop=True)\n",
    "#         print(order_info_set)\n",
    "#       获取起航时间\n",
    "        start_time = order_info_set['timestamp'].min()\n",
    "        start_index = 0\n",
    "        for (index, info_item) in order_info_set.iterrows():\n",
    "            if abs(info_item['longitude']-start_longitude) < 0.5 and abs(info_item['latitude']-start_latitude) < 0.5 and info_item['speed'] > 0:\n",
    "                start_time = max(start_time, info_item['timestamp'])\n",
    "                start_index = index\n",
    "                break \n",
    "#       获取到达目的地时间，这里需要用 GPS 判断\n",
    "        end_time = order_info_set['timestamp'].max()\n",
    "        end_index = order_info_set.size-1\n",
    "        for (index, info_item) in order_info_set.iterrows():\n",
    "            if abs(info_item['longitude'] - dest_longitude) < 0.3 and abs(info_item['latitude'] - dest_latitude) < 0.3:\n",
    "                end_time = min(end_time, info_item['timestamp'])\n",
    "                end_index = index\n",
    "                break\n",
    "                \n",
    "#         修正起点终点逆序\n",
    "        if (end_time < start_time):\n",
    "            start_time,end_time = end_time,start_time\n",
    "            start_index,end_index = end_index,start_index\n",
    "#         print(start_index, end_index)\n",
    "#         print(order_info_set)\n",
    "#         人工截取前 40% 的数据   \n",
    "        order_info_set = order_info_set[start_index:end_index+1]\n",
    "        cut_size = math.ceil(order_info_set.shape[0]*0.4)\n",
    "        order_info_set = order_info_set[0:cut_size]\n",
    "        \n",
    "#         截取数据\n",
    "        if (order_info_set.shape[0] > 100):\n",
    "            index = np.linspace(0, order_info_set.shape[0]-1, num=100,dtype=int).tolist()\n",
    "            order_info_set = order_info_set.iloc[index]     \n",
    "#         获取经纬度速度信息\n",
    "        agg_function = ['min', 'max', 'mean', 'median']\n",
    "        agg_col = ['latitude', 'longitude', 'speed']\n",
    "        feature_temp = order_info_set.groupby('loadingOrder')[agg_col].agg(agg_function).reset_index()\n",
    "        feature_temp.columns = ['loadingOrder'] + ['{}_{}'.format(i, j) for i in agg_col for j in agg_function]\n",
    "#         算出航行用时\n",
    "        feature_temp['label'] = (end_time - start_time).total_seconds()\n",
    "        train_data = pd.concat([train_data,feature_temp])\n",
    "    print('train data size     ', train_data.shape)\n",
    "    if (train_data.shape[0] < 10):\n",
    "        for i in range(5):\n",
    "            train_data = pd.concat([train_data,train_data])\n",
    "    return train_data.reset_index(drop=True)\n",
    "\n",
    "def get_test_data(order):\n",
    "    order_info_set = test_data_origin[test_data_origin['loadingOrder'] == order].sort_values(by='timestamp')\n",
    "    agg_function = ['min', 'max', 'mean', 'median']\n",
    "    agg_col = ['latitude', 'longitude', 'speed']\n",
    "    feature = order_info_set.groupby('loadingOrder')[agg_col].agg(agg_function).reset_index()\n",
    "    feature.columns = ['loadingOrder'] + ['{}_{}'.format(i, j) for i in agg_col for j in agg_function]\n",
    "    return feature.reset_index(drop=True)\n",
    "def mse_score_eval(preds, valid):\n",
    "    labels = valid.get_label()\n",
    "    scores = mean_squared_error(y_true=labels, y_pred=preds)\n",
    "    return 'mse_score', scores, True\n",
    "def train_model(x, y, seed=981125, is_shuffle=True):\n",
    "    train_pred = np.zeros((x.shape[0], ))\n",
    "    n_splits = min(5, x.shape[0])\n",
    "    # Kfold\n",
    "    fold = KFold(n_splits=n_splits, shuffle=is_shuffle, random_state=seed)\n",
    "    kf_way = fold.split(x)\n",
    "    # params\n",
    "    params = {\n",
    "        'learning_rate': 0.01,\n",
    "        'boosting_type': 'gbdt',\n",
    "        'objective': 'regression',\n",
    "        'num_leaves': 36,\n",
    "        'feature_fraction': 0.6,\n",
    "        'bagging_fraction': 0.7,\n",
    "        'bagging_freq': 6,\n",
    "        'seed': 8,\n",
    "        'bagging_seed': 1,\n",
    "        'feature_fraction_seed': 7,\n",
    "        'min_data_in_leaf': 25,\n",
    "        'nthread': 8,\n",
    "        'verbose': 1,\n",
    "    }\n",
    "    # train\n",
    "    for n_fold, (train_idx, valid_idx) in enumerate(kf_way, start=1):\n",
    "        train_x, train_y = x.iloc[train_idx], y.iloc[train_idx]\n",
    "        valid_x, valid_y = x.iloc[valid_idx], y.iloc[valid_idx]\n",
    "        # 数据加载\n",
    "        n_train = lgb.Dataset(train_x, label=train_y)\n",
    "        n_valid = lgb.Dataset(valid_x, label=valid_y)\n",
    "        clf = lgb.train(\n",
    "            params=params,\n",
    "            train_set=n_train,\n",
    "            num_boost_round=3000,\n",
    "            valid_sets=[n_valid],\n",
    "            early_stopping_rounds=100,\n",
    "            verbose_eval=100,\n",
    "            feval=mse_score_eval\n",
    "        )\n",
    "        train_pred[valid_idx] = clf.predict(valid_x, num_iteration=clf.best_iteration)\n",
    "    return clf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/22 [00:00<?, ?it/s]\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      "1it [00:04,  4.76s/it]\u001b[A\n",
      "2it [00:09,  4.75s/it]\u001b[A\n",
      "3it [00:14,  4.70s/it]\u001b[A\n",
      "4it [00:18,  4.68s/it]\u001b[A\n",
      "5it [00:23,  4.69s/it]\u001b[A\n",
      "6it [00:28,  4.79s/it]\u001b[A\n",
      "7it [00:32,  4.71s/it]\u001b[A\n",
      "8it [00:37,  4.73s/it]\u001b[A\n",
      "9it [00:42,  4.85s/it]\u001b[A\n",
      "10it [00:48,  5.15s/it]\u001b[A\n",
      "11it [00:53,  5.01s/it]\u001b[A\n",
      "12it [00:58,  5.16s/it]\u001b[A\n",
      "13it [01:03,  5.01s/it]\u001b[A\n",
      "14it [01:09,  5.16s/it]\u001b[A\n",
      "15it [01:14,  5.11s/it]\u001b[A\n",
      "16it [01:19,  5.16s/it]\u001b[A\n",
      "17it [01:24,  5.06s/it]\u001b[A\n",
      "18it [01:29,  5.05s/it]\u001b[A\n",
      "19it [01:34,  5.02s/it]\u001b[A\n",
      "20it [01:38,  4.91s/it]\u001b[A\n",
      "21it [01:43,  4.87s/it]\u001b[A\n",
      "22it [01:48,  4.98s/it]\u001b[A\n",
      "23it [02:02,  7.73s/it]\u001b[A\n",
      "24it [02:08,  7.09s/it]\u001b[A\n",
      "25it [02:14,  6.73s/it]\u001b[A\n",
      "26it [02:19,  6.36s/it]\u001b[A\n",
      "27it [02:25,  6.07s/it]\u001b[A\n",
      "28it [02:31,  5.97s/it]\u001b[A\n",
      "29it [02:37,  6.10s/it]\u001b[A\n",
      "30it [02:43,  5.99s/it]\u001b[A\n",
      "31it [02:48,  5.71s/it]\u001b[A\n",
      "32it [02:53,  5.68s/it]\u001b[A\n",
      "33it [02:59,  5.77s/it]\u001b[A\n",
      "34it [03:05,  5.75s/it]\u001b[A\n",
      "35it [03:11,  5.93s/it]\u001b[A\n",
      "36it [03:17,  5.87s/it]\u001b[A\n",
      "37it [03:23,  5.86s/it]\u001b[A\n",
      "38it [03:29,  5.94s/it]\u001b[A\n",
      "39it [03:35,  5.86s/it]\u001b[A\n",
      "40it [03:40,  5.77s/it]\u001b[A\n",
      "41it [03:46,  5.82s/it]\u001b[A\n",
      "42it [03:53,  5.98s/it]\u001b[A\n",
      "43it [03:59,  6.08s/it]\u001b[A\n",
      "44it [04:05,  6.13s/it]\u001b[A\n",
      "45it [04:11,  6.10s/it]\u001b[A\n",
      "46it [04:17,  6.12s/it]\u001b[A\n",
      "47it [04:23,  6.11s/it]\u001b[A\n",
      "48it [04:30,  6.16s/it]\u001b[A\n",
      "49it [04:36,  6.14s/it]\u001b[A\n",
      "50it [04:42,  6.20s/it]\u001b[A\n",
      "51it [04:49,  6.29s/it]\u001b[A\n",
      "52it [04:55,  6.32s/it]\u001b[A\n",
      "53it [05:02,  6.44s/it]\u001b[A\n",
      "54it [05:08,  6.45s/it]\u001b[A\n",
      "55it [05:15,  6.46s/it]\u001b[A\n",
      "56it [05:22,  6.72s/it]\u001b[A\n",
      "57it [05:28,  6.60s/it]\u001b[A\n",
      "58it [05:35,  6.62s/it]\u001b[A\n",
      "59it [05:42,  6.59s/it]\u001b[A\n",
      "60it [05:48,  6.61s/it]\u001b[A\n",
      "61it [05:55,  6.64s/it]\u001b[A\n",
      "62it [06:02,  6.72s/it]\u001b[A\n",
      "63it [06:09,  6.81s/it]\u001b[A\n",
      "64it [06:16,  6.88s/it]\u001b[A\n",
      "65it [06:23,  7.05s/it]\u001b[A\n",
      "66it [06:31,  7.14s/it]\u001b[A\n",
      "67it [06:38,  7.16s/it]\u001b[A\n",
      "68it [06:45,  7.18s/it]\u001b[A\n",
      "69it [06:53,  7.30s/it]\u001b[A\n",
      "70it [06:56,  5.96s/it]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/493 [00:00<?, ?it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CNYTN-MXZLO (493,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 1/493 [00:02<22:59,  2.80s/it]\u001b[A\n",
      "  0%|          | 2/493 [00:03<18:51,  2.31s/it]\u001b[A\n",
      "  1%|          | 3/493 [00:05<16:18,  2.00s/it]\u001b[A\n",
      "  1%|          | 4/493 [00:06<14:18,  1.76s/it]\u001b[A\n",
      "  1%|          | 5/493 [00:07<12:45,  1.57s/it]\u001b[A\n",
      "  1%|          | 6/493 [00:08<11:43,  1.44s/it]\u001b[A\n",
      "  1%|▏         | 7/493 [00:09<10:47,  1.33s/it]\u001b[A\n",
      "  2%|▏         | 8/493 [00:11<10:43,  1.33s/it]\u001b[A\n",
      "  2%|▏         | 9/493 [00:12<11:11,  1.39s/it]\u001b[A\n",
      "  2%|▏         | 10/493 [00:13<10:41,  1.33s/it]\u001b[A\n",
      "  2%|▏         | 11/493 [00:14<10:09,  1.27s/it]\u001b[A\n",
      "  2%|▏         | 12/493 [00:16<09:48,  1.22s/it]\u001b[A\n",
      "  3%|▎         | 13/493 [00:17<09:31,  1.19s/it]\u001b[A\n",
      "  3%|▎         | 14/493 [00:18<09:15,  1.16s/it]\u001b[A\n",
      "  3%|▎         | 15/493 [00:19<09:12,  1.16s/it]\u001b[A\n",
      "  3%|▎         | 16/493 [00:20<09:05,  1.14s/it]\u001b[A\n",
      "  3%|▎         | 17/493 [00:21<09:24,  1.18s/it]\u001b[A\n",
      "  4%|▎         | 18/493 [00:22<09:22,  1.18s/it]\u001b[A\n",
      "  4%|▍         | 19/493 [00:24<09:14,  1.17s/it]\u001b[A\n",
      "  4%|▍         | 20/493 [00:25<09:07,  1.16s/it]\u001b[A\n",
      "  4%|▍         | 21/493 [00:26<08:59,  1.14s/it]\u001b[A\n",
      "  4%|▍         | 22/493 [00:27<08:56,  1.14s/it]\u001b[A\n",
      "  5%|▍         | 23/493 [00:28<08:49,  1.13s/it]\u001b[A\n",
      "  5%|▍         | 24/493 [00:29<08:47,  1.13s/it]\u001b[A\n",
      "  5%|▌         | 25/493 [00:34<18:23,  2.36s/it]\u001b[A\n",
      "  5%|▌         | 26/493 [00:36<15:25,  1.98s/it]\u001b[A\n",
      "  5%|▌         | 27/493 [00:37<13:29,  1.74s/it]\u001b[A\n",
      "  6%|▌         | 28/493 [00:38<12:08,  1.57s/it]\u001b[A\n",
      "  6%|▌         | 29/493 [00:39<11:08,  1.44s/it]\u001b[A\n",
      "  6%|▌         | 30/493 [00:40<10:41,  1.39s/it]\u001b[A\n",
      "  6%|▋         | 31/493 [00:41<10:07,  1.31s/it]\u001b[A\n",
      "  6%|▋         | 32/493 [00:43<09:38,  1.25s/it]\u001b[A\n",
      "  7%|▋         | 33/493 [00:44<09:14,  1.20s/it]\u001b[A\n",
      "  7%|▋         | 34/493 [00:45<08:57,  1.17s/it]\u001b[A\n",
      "  7%|▋         | 35/493 [00:46<08:40,  1.14s/it]\u001b[A\n",
      "  7%|▋         | 36/493 [00:47<08:34,  1.13s/it]\u001b[A\n",
      "  8%|▊         | 37/493 [00:48<08:27,  1.11s/it]\u001b[A\n",
      "  8%|▊         | 38/493 [00:49<08:30,  1.12s/it]\u001b[A\n",
      "  8%|▊         | 39/493 [00:50<08:24,  1.11s/it]\u001b[A\n",
      "  8%|▊         | 40/493 [00:51<08:21,  1.11s/it]\u001b[A\n",
      "  8%|▊         | 41/493 [00:52<08:31,  1.13s/it]\u001b[A\n",
      "  9%|▊         | 42/493 [00:54<08:39,  1.15s/it]\u001b[A\n",
      "  9%|▊         | 43/493 [00:55<08:29,  1.13s/it]\u001b[A\n",
      "  9%|▉         | 44/493 [00:58<14:05,  1.88s/it]\u001b[A\n",
      "  9%|▉         | 45/493 [01:01<14:59,  2.01s/it]\u001b[A\n",
      "  9%|▉         | 46/493 [01:07<24:11,  3.25s/it]\u001b[A\n",
      " 10%|▉         | 47/493 [01:09<22:35,  3.04s/it]\u001b[A\n",
      " 10%|▉         | 48/493 [01:11<18:15,  2.46s/it]\u001b[A\n",
      " 10%|▉         | 49/493 [01:12<15:09,  2.05s/it]\u001b[A\n",
      " 10%|█         | 50/493 [01:13<13:02,  1.77s/it]\u001b[A\n",
      " 10%|█         | 51/493 [01:14<11:32,  1.57s/it]\u001b[A\n",
      " 11%|█         | 52/493 [01:15<10:26,  1.42s/it]\u001b[A\n",
      " 11%|█         | 53/493 [01:20<19:05,  2.60s/it]\u001b[A\n",
      " 11%|█         | 54/493 [01:26<25:20,  3.46s/it]\u001b[A\n",
      " 11%|█         | 55/493 [01:31<29:04,  3.98s/it]\u001b[A\n",
      " 11%|█▏        | 56/493 [01:38<35:05,  4.82s/it]\u001b[A\n",
      " 12%|█▏        | 57/493 [01:44<39:07,  5.39s/it]\u001b[A\n",
      " 12%|█▏        | 58/493 [01:50<40:18,  5.56s/it]\u001b[A\n",
      " 12%|█▏        | 59/493 [01:57<41:53,  5.79s/it]\u001b[A\n",
      " 12%|█▏        | 60/493 [02:03<42:48,  5.93s/it]\u001b[A\n",
      " 12%|█▏        | 61/493 [02:08<41:41,  5.79s/it]\u001b[A\n",
      " 13%|█▎        | 62/493 [02:14<40:17,  5.61s/it]\u001b[A\n",
      " 13%|█▎        | 63/493 [02:19<39:14,  5.48s/it]\u001b[A\n",
      " 13%|█▎        | 64/493 [02:20<29:42,  4.16s/it]\u001b[A\n",
      " 13%|█▎        | 65/493 [02:21<22:58,  3.22s/it]\u001b[A\n",
      " 13%|█▎        | 66/493 [02:23<21:06,  2.97s/it]\u001b[A\n",
      " 14%|█▎        | 67/493 [02:24<16:54,  2.38s/it]\u001b[A\n",
      " 14%|█▍        | 68/493 [02:25<13:56,  1.97s/it]\u001b[A\n",
      " 14%|█▍        | 69/493 [02:26<11:46,  1.67s/it]\u001b[A\n",
      " 14%|█▍        | 70/493 [02:29<13:23,  1.90s/it]\u001b[A\n",
      " 14%|█▍        | 71/493 [02:31<14:21,  2.04s/it]\u001b[A\n",
      " 15%|█▍        | 72/493 [02:32<12:11,  1.74s/it]\u001b[A\n",
      " 15%|█▍        | 73/493 [02:33<10:33,  1.51s/it]\u001b[A\n",
      " 15%|█▌        | 74/493 [02:34<09:29,  1.36s/it]\u001b[A\n",
      " 15%|█▌        | 75/493 [02:35<08:43,  1.25s/it]\u001b[A\n",
      " 15%|█▌        | 76/493 [02:36<08:09,  1.17s/it]\u001b[A\n",
      " 16%|█▌        | 77/493 [02:38<10:29,  1.51s/it]\u001b[A\n",
      " 16%|█▌        | 78/493 [02:39<09:24,  1.36s/it]\u001b[A\n",
      " 16%|█▌        | 79/493 [02:40<08:48,  1.28s/it]\u001b[A\n",
      " 16%|█▌        | 80/493 [02:41<08:13,  1.19s/it]\u001b[A\n",
      " 16%|█▋        | 81/493 [02:43<07:58,  1.16s/it]\u001b[A\n",
      " 17%|█▋        | 82/493 [02:44<07:46,  1.14s/it]\u001b[A\n",
      " 17%|█▋        | 83/493 [02:45<07:47,  1.14s/it]\u001b[A\n",
      " 17%|█▋        | 84/493 [02:46<07:32,  1.11s/it]\u001b[A\n",
      " 17%|█▋        | 85/493 [02:48<09:48,  1.44s/it]\u001b[A\n",
      " 17%|█▋        | 86/493 [02:49<09:08,  1.35s/it]\u001b[A\n",
      " 18%|█▊        | 87/493 [02:50<08:57,  1.32s/it]\u001b[A\n",
      " 18%|█▊        | 88/493 [02:52<08:29,  1.26s/it]\u001b[A\n",
      " 18%|█▊        | 89/493 [02:53<08:12,  1.22s/it]\u001b[A\n",
      " 18%|█▊        | 90/493 [02:54<08:05,  1.20s/it]\u001b[A\n",
      " 18%|█▊        | 91/493 [02:55<07:51,  1.17s/it]\u001b[A\n",
      " 19%|█▊        | 92/493 [02:56<07:54,  1.18s/it]\u001b[A\n",
      " 19%|█▉        | 93/493 [02:57<07:40,  1.15s/it]\u001b[A\n",
      " 19%|█▉        | 94/493 [02:58<07:30,  1.13s/it]\u001b[A\n",
      " 19%|█▉        | 95/493 [02:59<07:28,  1.13s/it]\u001b[A\n",
      " 19%|█▉        | 96/493 [03:01<07:36,  1.15s/it]\u001b[A\n",
      " 20%|█▉        | 97/493 [03:02<07:41,  1.17s/it]\u001b[A\n",
      " 20%|█▉        | 98/493 [03:03<07:40,  1.17s/it]\u001b[A\n",
      " 20%|██        | 99/493 [03:04<07:35,  1.16s/it]\u001b[A\n",
      " 20%|██        | 100/493 [03:05<07:45,  1.19s/it]\u001b[A\n",
      " 20%|██        | 101/493 [03:07<07:52,  1.21s/it]\u001b[A\n",
      " 21%|██        | 102/493 [03:08<07:48,  1.20s/it]\u001b[A\n",
      " 21%|██        | 103/493 [03:09<07:49,  1.20s/it]\u001b[A\n",
      " 21%|██        | 104/493 [03:11<09:03,  1.40s/it]\u001b[A\n",
      " 21%|██▏       | 105/493 [03:12<08:35,  1.33s/it]\u001b[A\n",
      " 22%|██▏       | 106/493 [03:15<11:23,  1.77s/it]\u001b[A\n",
      " 22%|██▏       | 107/493 [03:16<10:13,  1.59s/it]\u001b[A\n",
      " 22%|██▏       | 108/493 [03:19<13:20,  2.08s/it]\u001b[A\n",
      " 22%|██▏       | 109/493 [03:20<11:30,  1.80s/it]\u001b[A\n",
      " 22%|██▏       | 110/493 [03:21<09:38,  1.51s/it]\u001b[A\n",
      " 23%|██▎       | 111/493 [03:22<08:19,  1.31s/it]\u001b[A\n",
      " 23%|██▎       | 112/493 [03:23<07:29,  1.18s/it]\u001b[A\n",
      " 23%|██▎       | 113/493 [03:24<07:11,  1.13s/it]\u001b[A\n",
      " 23%|██▎       | 114/493 [03:25<06:59,  1.11s/it]\u001b[A\n",
      " 23%|██▎       | 115/493 [03:26<06:34,  1.04s/it]\u001b[A\n",
      " 24%|██▎       | 116/493 [03:27<06:15,  1.00it/s]\u001b[A\n",
      " 24%|██▎       | 117/493 [03:28<06:11,  1.01it/s]\u001b[A\n",
      " 24%|██▍       | 118/493 [03:29<06:05,  1.03it/s]\u001b[A\n",
      " 24%|██▍       | 119/493 [03:30<06:02,  1.03it/s]\u001b[A\n",
      " 24%|██▍       | 120/493 [03:31<05:59,  1.04it/s]\u001b[A\n",
      " 25%|██▍       | 121/493 [03:32<05:57,  1.04it/s]\u001b[A\n",
      " 25%|██▍       | 122/493 [03:33<06:06,  1.01it/s]\u001b[A\n",
      " 25%|██▍       | 123/493 [03:34<06:05,  1.01it/s]\u001b[A\n",
      " 25%|██▌       | 124/493 [03:35<06:04,  1.01it/s]\u001b[A\n",
      " 25%|██▌       | 125/493 [03:36<06:01,  1.02it/s]\u001b[A\n",
      " 26%|██▌       | 126/493 [03:37<06:04,  1.01it/s]\u001b[A\n",
      " 26%|██▌       | 127/493 [03:38<06:05,  1.00it/s]\u001b[A\n",
      " 26%|██▌       | 128/493 [03:39<06:04,  1.00it/s]\u001b[A\n",
      " 26%|██▌       | 129/493 [03:40<06:05,  1.00s/it]\u001b[A\n",
      " 26%|██▋       | 130/493 [03:41<06:00,  1.01it/s]\u001b[A\n",
      " 27%|██▋       | 131/493 [03:42<05:59,  1.01it/s]\u001b[A\n",
      " 27%|██▋       | 132/493 [03:43<05:59,  1.01it/s]\u001b[A\n",
      " 27%|██▋       | 133/493 [03:44<06:08,  1.02s/it]\u001b[A\n",
      " 27%|██▋       | 134/493 [03:45<06:05,  1.02s/it]\u001b[A\n",
      " 27%|██▋       | 135/493 [03:46<06:04,  1.02s/it]\u001b[A\n",
      " 28%|██▊       | 136/493 [03:47<06:04,  1.02s/it]\u001b[A\n",
      " 28%|██▊       | 137/493 [03:48<06:02,  1.02s/it]\u001b[A\n",
      " 28%|██▊       | 138/493 [03:49<06:55,  1.17s/it]\u001b[A\n",
      " 28%|██▊       | 139/493 [03:51<07:11,  1.22s/it]\u001b[A\n",
      " 28%|██▊       | 140/493 [03:52<07:22,  1.25s/it]\u001b[A\n",
      " 29%|██▊       | 141/493 [03:53<07:35,  1.29s/it]\u001b[A\n",
      " 29%|██▉       | 142/493 [03:55<07:39,  1.31s/it]\u001b[A\n",
      " 29%|██▉       | 143/493 [03:56<08:10,  1.40s/it]\u001b[A\n",
      " 29%|██▉       | 144/493 [03:58<08:01,  1.38s/it]\u001b[A\n",
      " 29%|██▉       | 145/493 [03:59<08:00,  1.38s/it]\u001b[A\n",
      " 30%|██▉       | 146/493 [04:00<07:55,  1.37s/it]\u001b[A\n",
      " 30%|██▉       | 147/493 [04:02<07:52,  1.37s/it]\u001b[A\n",
      " 30%|███       | 148/493 [04:03<07:49,  1.36s/it]\u001b[A\n",
      " 30%|███       | 149/493 [04:05<08:43,  1.52s/it]\u001b[A\n",
      " 30%|███       | 150/493 [04:06<08:46,  1.53s/it]\u001b[A\n",
      " 31%|███       | 151/493 [04:08<08:07,  1.43s/it]\u001b[A\n",
      " 31%|███       | 152/493 [04:09<07:44,  1.36s/it]\u001b[A\n",
      " 31%|███       | 153/493 [04:10<08:06,  1.43s/it]\u001b[A\n",
      " 31%|███       | 154/493 [04:12<07:53,  1.40s/it]\u001b[A\n",
      " 31%|███▏      | 155/493 [04:13<07:20,  1.30s/it]\u001b[A\n",
      " 32%|███▏      | 156/493 [04:14<06:43,  1.20s/it]\u001b[A\n",
      " 32%|███▏      | 157/493 [04:15<06:11,  1.11s/it]\u001b[A\n",
      " 32%|███▏      | 158/493 [04:16<05:59,  1.07s/it]\u001b[A\n",
      " 32%|███▏      | 159/493 [04:17<05:36,  1.01s/it]\u001b[A\n",
      " 32%|███▏      | 160/493 [04:17<05:32,  1.00it/s]\u001b[A\n",
      " 33%|███▎      | 161/493 [04:20<07:25,  1.34s/it]\u001b[A\n",
      " 33%|███▎      | 162/493 [04:22<09:15,  1.68s/it]\u001b[A\n",
      " 33%|███▎      | 163/493 [04:24<09:21,  1.70s/it]\u001b[A\n",
      " 33%|███▎      | 164/493 [04:26<09:18,  1.70s/it]\u001b[A\n",
      " 33%|███▎      | 165/493 [04:27<08:10,  1.49s/it]\u001b[A\n",
      " 34%|███▎      | 166/493 [04:28<07:29,  1.37s/it]\u001b[A\n",
      " 34%|███▍      | 167/493 [04:29<06:56,  1.28s/it]\u001b[A\n",
      " 34%|███▍      | 168/493 [04:31<08:22,  1.55s/it]\u001b[A\n",
      " 34%|███▍      | 169/493 [04:33<09:35,  1.78s/it]\u001b[A\n",
      " 34%|███▍      | 170/493 [04:35<10:11,  1.89s/it]\u001b[A\n",
      " 35%|███▍      | 171/493 [04:37<10:28,  1.95s/it]\u001b[A\n",
      " 35%|███▍      | 172/493 [04:39<10:28,  1.96s/it]\u001b[A\n",
      " 35%|███▌      | 173/493 [04:41<10:17,  1.93s/it]\u001b[A\n",
      " 35%|███▌      | 174/493 [04:43<10:08,  1.91s/it]\u001b[A\n",
      " 35%|███▌      | 175/493 [04:45<10:05,  1.90s/it]\u001b[A\n",
      " 36%|███▌      | 176/493 [04:47<10:00,  1.90s/it]\u001b[A\n",
      " 36%|███▌      | 177/493 [04:49<09:56,  1.89s/it]\u001b[A\n",
      " 36%|███▌      | 178/493 [04:51<10:09,  1.93s/it]\u001b[A\n",
      " 36%|███▋      | 179/493 [04:53<10:39,  2.04s/it]\u001b[A\n",
      " 37%|███▋      | 180/493 [04:55<10:27,  2.00s/it]\u001b[A\n",
      " 37%|███▋      | 181/493 [04:57<09:54,  1.90s/it]\u001b[A\n",
      " 37%|███▋      | 182/493 [04:58<09:37,  1.86s/it]\u001b[A\n",
      " 37%|███▋      | 183/493 [05:00<09:03,  1.75s/it]\u001b[A\n",
      " 37%|███▋      | 184/493 [05:02<08:51,  1.72s/it]\u001b[A\n",
      " 38%|███▊      | 185/493 [05:03<08:28,  1.65s/it]\u001b[A\n",
      " 38%|███▊      | 186/493 [05:05<08:09,  1.60s/it]\u001b[A\n",
      " 38%|███▊      | 187/493 [05:06<08:04,  1.58s/it]\u001b[A\n",
      " 38%|███▊      | 188/493 [05:08<08:03,  1.58s/it]\u001b[A\n",
      " 38%|███▊      | 189/493 [05:09<07:59,  1.58s/it]\u001b[A\n",
      " 39%|███▊      | 190/493 [05:11<07:56,  1.57s/it]\u001b[A\n",
      " 39%|███▊      | 191/493 [05:12<07:37,  1.51s/it]\u001b[A\n",
      " 39%|███▉      | 192/493 [05:14<07:19,  1.46s/it]\u001b[A\n",
      " 39%|███▉      | 193/493 [05:15<07:28,  1.49s/it]\u001b[A\n",
      " 39%|███▉      | 194/493 [05:17<07:19,  1.47s/it]\u001b[A\n",
      " 40%|███▉      | 195/493 [05:18<07:10,  1.44s/it]\u001b[A\n",
      " 40%|███▉      | 196/493 [05:19<06:24,  1.29s/it]\u001b[A\n",
      " 40%|███▉      | 197/493 [05:20<05:49,  1.18s/it]\u001b[A\n",
      " 40%|████      | 198/493 [05:21<05:24,  1.10s/it]\u001b[A\n",
      " 40%|████      | 199/493 [05:22<05:06,  1.04s/it]\u001b[A\n",
      " 41%|████      | 200/493 [05:22<04:51,  1.00it/s]\u001b[A\n",
      " 41%|████      | 201/493 [05:23<04:45,  1.02it/s]\u001b[A\n",
      " 41%|████      | 202/493 [05:24<04:52,  1.01s/it]\u001b[A\n",
      " 41%|████      | 203/493 [05:25<04:45,  1.02it/s]\u001b[A\n",
      " 41%|████▏     | 204/493 [05:26<04:37,  1.04it/s]\u001b[A\n",
      " 42%|████▏     | 205/493 [05:27<04:30,  1.06it/s]\u001b[A\n",
      " 42%|████▏     | 206/493 [05:28<04:26,  1.08it/s]\u001b[A\n",
      " 42%|████▏     | 207/493 [05:29<04:22,  1.09it/s]\u001b[A\n",
      " 42%|████▏     | 208/493 [05:30<04:20,  1.10it/s]\u001b[A\n",
      " 42%|████▏     | 209/493 [05:31<04:26,  1.06it/s]\u001b[A\n",
      " 43%|████▎     | 210/493 [05:32<04:31,  1.04it/s]\u001b[A\n",
      " 43%|████▎     | 211/493 [05:33<04:27,  1.06it/s]\u001b[A\n",
      " 43%|████▎     | 212/493 [05:34<04:27,  1.05it/s]\u001b[A\n",
      " 43%|████▎     | 213/493 [05:35<04:40,  1.00s/it]\u001b[A\n",
      " 43%|████▎     | 214/493 [05:36<04:37,  1.01it/s]\u001b[A\n",
      " 44%|████▎     | 215/493 [05:37<04:28,  1.04it/s]\u001b[A\n",
      " 44%|████▍     | 216/493 [05:38<04:22,  1.06it/s]\u001b[A\n",
      " 44%|████▍     | 217/493 [05:39<04:18,  1.07it/s]\u001b[A\n",
      " 44%|████▍     | 218/493 [05:40<04:17,  1.07it/s]\u001b[A\n",
      " 44%|████▍     | 219/493 [05:41<04:20,  1.05it/s]\u001b[A\n",
      " 45%|████▍     | 220/493 [05:41<04:20,  1.05it/s]\u001b[A\n",
      " 45%|████▍     | 221/493 [05:43<04:26,  1.02it/s]\u001b[A\n",
      " 45%|████▌     | 222/493 [05:44<04:29,  1.01it/s]\u001b[A\n",
      " 45%|████▌     | 223/493 [05:45<04:35,  1.02s/it]\u001b[A\n",
      " 45%|████▌     | 224/493 [05:46<04:36,  1.03s/it]\u001b[A\n",
      " 46%|████▌     | 225/493 [05:47<04:35,  1.03s/it]\u001b[A\n",
      " 46%|████▌     | 226/493 [05:48<04:38,  1.04s/it]\u001b[A\n",
      " 46%|████▌     | 227/493 [05:49<04:40,  1.05s/it]\u001b[A\n",
      " 46%|████▌     | 228/493 [05:50<04:31,  1.02s/it]\u001b[A\n",
      " 46%|████▋     | 229/493 [05:51<04:25,  1.01s/it]\u001b[A\n",
      " 47%|████▋     | 230/493 [05:52<04:26,  1.01s/it]\u001b[A\n",
      " 47%|████▋     | 231/493 [05:53<04:46,  1.09s/it]\u001b[A\n",
      " 47%|████▋     | 232/493 [05:54<04:59,  1.15s/it]\u001b[A\n",
      " 47%|████▋     | 233/493 [05:56<05:05,  1.18s/it]\u001b[A\n",
      " 47%|████▋     | 234/493 [05:57<05:17,  1.22s/it]\u001b[A\n",
      " 48%|████▊     | 235/493 [05:58<05:06,  1.19s/it]\u001b[A\n",
      " 48%|████▊     | 236/493 [05:59<04:42,  1.10s/it]\u001b[A\n",
      " 48%|████▊     | 237/493 [06:00<04:25,  1.04s/it]\u001b[A\n",
      " 48%|████▊     | 238/493 [06:01<04:15,  1.00s/it]\u001b[A\n",
      " 48%|████▊     | 239/493 [06:02<04:07,  1.03it/s]\u001b[A\n",
      " 49%|████▊     | 240/493 [06:03<03:58,  1.06it/s]\u001b[A\n",
      " 49%|████▉     | 241/493 [06:04<04:04,  1.03it/s]\u001b[A\n",
      " 49%|████▉     | 242/493 [06:05<04:08,  1.01it/s]\u001b[A\n",
      " 49%|████▉     | 243/493 [06:06<04:19,  1.04s/it]\u001b[A\n",
      " 49%|████▉     | 244/493 [06:07<04:34,  1.10s/it]\u001b[A\n",
      " 50%|████▉     | 245/493 [06:08<04:39,  1.13s/it]\u001b[A\n",
      " 50%|████▉     | 246/493 [06:09<04:34,  1.11s/it]\u001b[A\n",
      " 50%|█████     | 247/493 [06:10<04:31,  1.10s/it]\u001b[A\n",
      " 50%|█████     | 248/493 [06:11<04:26,  1.09s/it]\u001b[A\n",
      " 51%|█████     | 249/493 [06:12<04:23,  1.08s/it]\u001b[A\n",
      " 51%|█████     | 250/493 [06:14<04:19,  1.07s/it]\u001b[A\n",
      " 51%|█████     | 251/493 [06:14<04:12,  1.04s/it]\u001b[A\n",
      " 51%|█████     | 252/493 [06:16<04:11,  1.04s/it]\u001b[A\n",
      " 51%|█████▏    | 253/493 [06:17<04:08,  1.03s/it]\u001b[A\n",
      " 52%|█████▏    | 254/493 [06:18<04:05,  1.03s/it]\u001b[A\n",
      " 52%|█████▏    | 255/493 [06:19<04:04,  1.03s/it]\u001b[A\n",
      " 52%|█████▏    | 256/493 [06:20<03:59,  1.01s/it]\u001b[A\n",
      " 52%|█████▏    | 257/493 [06:21<03:57,  1.01s/it]\u001b[A\n",
      " 52%|█████▏    | 258/493 [06:22<03:58,  1.02s/it]\u001b[A\n",
      " 53%|█████▎    | 259/493 [06:23<03:57,  1.02s/it]\u001b[A\n",
      " 53%|█████▎    | 260/493 [06:24<03:57,  1.02s/it]\u001b[A\n",
      " 53%|█████▎    | 261/493 [06:25<03:56,  1.02s/it]\u001b[A\n",
      " 53%|█████▎    | 262/493 [06:26<03:54,  1.01s/it]\u001b[A\n",
      " 53%|█████▎    | 263/493 [06:27<03:55,  1.02s/it]\u001b[A\n",
      " 54%|█████▎    | 264/493 [06:28<03:46,  1.01it/s]\u001b[A\n",
      " 54%|█████▍    | 265/493 [06:29<03:47,  1.00it/s]\u001b[A\n",
      " 54%|█████▍    | 266/493 [06:30<03:46,  1.00it/s]\u001b[A\n",
      " 54%|█████▍    | 267/493 [06:31<03:43,  1.01it/s]\u001b[A\n",
      " 54%|█████▍    | 268/493 [06:32<03:41,  1.02it/s]\u001b[A\n",
      " 55%|█████▍    | 269/493 [06:33<03:38,  1.02it/s]\u001b[A\n",
      " 55%|█████▍    | 270/493 [06:33<03:37,  1.03it/s]\u001b[A\n",
      " 55%|█████▍    | 271/493 [06:34<03:33,  1.04it/s]\u001b[A\n",
      " 55%|█████▌    | 272/493 [06:35<03:31,  1.05it/s]\u001b[A\n",
      " 55%|█████▌    | 273/493 [06:36<03:31,  1.04it/s]\u001b[A\n",
      " 56%|█████▌    | 274/493 [06:37<03:29,  1.04it/s]\u001b[A\n",
      " 56%|█████▌    | 275/493 [06:38<03:27,  1.05it/s]\u001b[A\n",
      " 56%|█████▌    | 276/493 [06:39<03:27,  1.05it/s]\u001b[A\n",
      " 56%|█████▌    | 277/493 [06:40<03:26,  1.05it/s]\u001b[A\n",
      " 56%|█████▋    | 278/493 [06:41<03:26,  1.04it/s]\u001b[A\n",
      " 57%|█████▋    | 279/493 [06:42<03:40,  1.03s/it]\u001b[A\n",
      " 57%|█████▋    | 280/493 [06:43<03:45,  1.06s/it]\u001b[A\n",
      " 57%|█████▋    | 281/493 [06:45<03:56,  1.12s/it]\u001b[A\n",
      " 57%|█████▋    | 282/493 [06:46<04:01,  1.14s/it]\u001b[A\n",
      " 57%|█████▋    | 283/493 [06:47<04:01,  1.15s/it]\u001b[A\n",
      " 58%|█████▊    | 284/493 [06:48<04:00,  1.15s/it]\u001b[A\n",
      " 58%|█████▊    | 285/493 [06:49<03:59,  1.15s/it]\u001b[A\n",
      " 58%|█████▊    | 286/493 [06:51<03:59,  1.16s/it]\u001b[A\n",
      " 58%|█████▊    | 287/493 [06:52<04:02,  1.18s/it]\u001b[A\n",
      " 58%|█████▊    | 288/493 [06:53<04:01,  1.18s/it]\u001b[A\n",
      " 59%|█████▊    | 289/493 [06:54<03:59,  1.17s/it]\u001b[A\n",
      " 59%|█████▉    | 290/493 [06:55<03:59,  1.18s/it]\u001b[A\n",
      " 59%|█████▉    | 291/493 [06:57<04:00,  1.19s/it]\u001b[A\n",
      " 59%|█████▉    | 292/493 [06:57<03:25,  1.02s/it]\u001b[A\n",
      " 59%|█████▉    | 293/493 [06:58<03:01,  1.10it/s]\u001b[A\n",
      " 60%|█████▉    | 294/493 [06:58<02:43,  1.21it/s]\u001b[A\n",
      " 60%|█████▉    | 295/493 [06:59<02:32,  1.30it/s]\u001b[A\n",
      " 60%|██████    | 296/493 [07:00<02:23,  1.38it/s]\u001b[A\n",
      " 60%|██████    | 297/493 [07:00<02:21,  1.39it/s]\u001b[A\n",
      " 60%|██████    | 298/493 [07:01<02:20,  1.38it/s]\u001b[A\n",
      " 61%|██████    | 299/493 [07:02<02:17,  1.41it/s]\u001b[A\n",
      " 61%|██████    | 300/493 [07:02<02:13,  1.45it/s]\u001b[A\n",
      " 61%|██████    | 301/493 [07:05<03:36,  1.13s/it]\u001b[A\n",
      " 61%|██████▏   | 302/493 [07:05<03:20,  1.05s/it]\u001b[A\n",
      " 61%|██████▏   | 303/493 [07:06<03:09,  1.00it/s]\u001b[A\n",
      " 62%|██████▏   | 304/493 [07:07<03:02,  1.04it/s]\u001b[A\n",
      " 62%|██████▏   | 305/493 [07:08<02:56,  1.07it/s]\u001b[A\n",
      " 62%|██████▏   | 306/493 [07:09<02:51,  1.09it/s]\u001b[A\n",
      " 62%|██████▏   | 307/493 [07:10<02:47,  1.11it/s]\u001b[A\n",
      " 62%|██████▏   | 308/493 [07:11<02:44,  1.12it/s]\u001b[A\n",
      " 63%|██████▎   | 309/493 [07:12<02:50,  1.08it/s]\u001b[A\n",
      " 63%|██████▎   | 310/493 [07:13<02:51,  1.07it/s]\u001b[A\n",
      " 63%|██████▎   | 311/493 [07:14<02:50,  1.07it/s]\u001b[A\n",
      " 63%|██████▎   | 312/493 [07:15<02:49,  1.07it/s]\u001b[A\n",
      " 63%|██████▎   | 313/493 [07:15<02:48,  1.07it/s]\u001b[A\n",
      " 64%|██████▎   | 314/493 [07:16<02:51,  1.04it/s]\u001b[A\n",
      " 64%|██████▍   | 315/493 [07:17<02:52,  1.03it/s]\u001b[A\n",
      " 64%|██████▍   | 316/493 [07:18<02:49,  1.04it/s]\u001b[A\n",
      " 64%|██████▍   | 317/493 [07:19<02:47,  1.05it/s]\u001b[A\n",
      " 65%|██████▍   | 318/493 [07:20<02:46,  1.05it/s]\u001b[A\n",
      " 65%|██████▍   | 319/493 [07:21<02:46,  1.04it/s]\u001b[A\n",
      " 65%|██████▍   | 320/493 [07:22<02:47,  1.03it/s]\u001b[A\n",
      " 65%|██████▌   | 321/493 [07:23<02:47,  1.02it/s]\u001b[A\n",
      " 65%|██████▌   | 322/493 [07:24<02:49,  1.01it/s]\u001b[A\n",
      " 66%|██████▌   | 323/493 [07:25<02:50,  1.00s/it]\u001b[A\n",
      " 66%|██████▌   | 324/493 [07:26<02:50,  1.01s/it]\u001b[A\n",
      " 66%|██████▌   | 325/493 [07:27<02:47,  1.01it/s]\u001b[A\n",
      " 66%|██████▌   | 326/493 [07:28<02:50,  1.02s/it]\u001b[A\n",
      " 66%|██████▋   | 327/493 [07:35<07:44,  2.80s/it]\u001b[A\n",
      " 67%|██████▋   | 328/493 [07:41<10:16,  3.74s/it]\u001b[A\n",
      " 67%|██████▋   | 329/493 [07:46<11:08,  4.07s/it]\u001b[A\n",
      " 67%|██████▋   | 330/493 [07:52<12:11,  4.49s/it]\u001b[A\n",
      " 67%|██████▋   | 331/493 [07:57<12:40,  4.69s/it]\u001b[A\n",
      " 67%|██████▋   | 332/493 [08:03<13:28,  5.02s/it]\u001b[A\n",
      " 68%|██████▊   | 333/493 [08:08<13:32,  5.08s/it]\u001b[A\n",
      " 68%|██████▊   | 334/493 [08:09<10:15,  3.87s/it]\u001b[A\n",
      " 68%|██████▊   | 335/493 [08:10<07:59,  3.04s/it]\u001b[A\n",
      " 68%|██████▊   | 336/493 [08:11<06:30,  2.48s/it]\u001b[A\n",
      " 68%|██████▊   | 337/493 [08:12<05:24,  2.08s/it]\u001b[A\n",
      " 69%|██████▊   | 338/493 [08:13<04:38,  1.80s/it]\u001b[A\n",
      " 69%|██████▉   | 339/493 [08:15<04:10,  1.63s/it]\u001b[A\n",
      " 69%|██████▉   | 340/493 [08:16<03:51,  1.51s/it]\u001b[A\n",
      " 69%|██████▉   | 341/493 [08:17<03:31,  1.39s/it]\u001b[A\n",
      " 69%|██████▉   | 342/493 [08:18<03:18,  1.32s/it]\u001b[A\n",
      " 70%|██████▉   | 343/493 [08:19<03:09,  1.27s/it]\u001b[A\n",
      " 70%|██████▉   | 344/493 [08:21<03:08,  1.26s/it]\u001b[A\n",
      " 70%|██████▉   | 345/493 [08:22<03:05,  1.25s/it]\u001b[A\n",
      " 70%|███████   | 346/493 [08:23<03:01,  1.23s/it]\u001b[A\n",
      " 70%|███████   | 347/493 [08:25<03:44,  1.54s/it]\u001b[A\n",
      " 71%|███████   | 348/493 [08:27<04:16,  1.77s/it]\u001b[A\n",
      " 71%|███████   | 349/493 [08:29<04:15,  1.77s/it]\u001b[A\n",
      " 71%|███████   | 350/493 [08:31<04:17,  1.80s/it]\u001b[A\n",
      " 71%|███████   | 351/493 [08:33<04:06,  1.73s/it]\u001b[A\n",
      " 71%|███████▏  | 352/493 [08:34<04:02,  1.72s/it]\u001b[A\n",
      " 72%|███████▏  | 353/493 [08:36<04:00,  1.72s/it]\u001b[A\n",
      " 72%|███████▏  | 354/493 [08:38<04:01,  1.73s/it]\u001b[A\n",
      " 72%|███████▏  | 355/493 [08:41<05:13,  2.27s/it]\u001b[A\n",
      " 72%|███████▏  | 356/493 [08:45<06:11,  2.71s/it]\u001b[A\n",
      " 72%|███████▏  | 357/493 [08:49<06:47,  2.99s/it]\u001b[A\n",
      " 73%|███████▎  | 358/493 [08:52<07:06,  3.16s/it]\u001b[A\n",
      " 73%|███████▎  | 359/493 [08:56<07:27,  3.34s/it]\u001b[A\n",
      " 73%|███████▎  | 360/493 [09:00<07:28,  3.37s/it]\u001b[A\n",
      " 73%|███████▎  | 361/493 [09:03<07:28,  3.40s/it]\u001b[A\n",
      " 73%|███████▎  | 362/493 [09:07<07:28,  3.43s/it]\u001b[A\n",
      " 74%|███████▎  | 363/493 [09:10<07:32,  3.48s/it]\u001b[A\n",
      " 74%|███████▍  | 364/493 [09:12<06:34,  3.06s/it]\u001b[A\n",
      " 74%|███████▍  | 365/493 [09:17<07:32,  3.54s/it]\u001b[A\n",
      " 74%|███████▍  | 366/493 [09:21<08:11,  3.87s/it]\u001b[A\n",
      " 74%|███████▍  | 367/493 [09:26<08:45,  4.17s/it]\u001b[A\n",
      " 75%|███████▍  | 368/493 [09:31<09:00,  4.32s/it]\u001b[A\n",
      " 75%|███████▍  | 369/493 [09:36<09:10,  4.44s/it]\u001b[A\n",
      " 75%|███████▌  | 370/493 [09:40<08:54,  4.34s/it]\u001b[A\n",
      " 75%|███████▌  | 371/493 [09:44<08:47,  4.32s/it]\u001b[A\n",
      " 75%|███████▌  | 372/493 [09:48<08:35,  4.26s/it]\u001b[A\n",
      " 76%|███████▌  | 373/493 [09:52<08:27,  4.23s/it]\u001b[A\n",
      " 76%|███████▌  | 374/493 [09:57<08:30,  4.29s/it]\u001b[A\n",
      " 76%|███████▌  | 375/493 [10:02<08:42,  4.43s/it]\u001b[A\n",
      " 76%|███████▋  | 376/493 [10:06<08:29,  4.35s/it]\u001b[A\n",
      " 76%|███████▋  | 377/493 [10:10<08:35,  4.44s/it]\u001b[A\n",
      " 77%|███████▋  | 378/493 [10:15<08:42,  4.54s/it]\u001b[A\n",
      " 77%|███████▋  | 379/493 [10:20<08:30,  4.48s/it]\u001b[A\n",
      " 77%|███████▋  | 380/493 [10:24<08:28,  4.50s/it]\u001b[A\n",
      " 77%|███████▋  | 381/493 [10:28<08:20,  4.47s/it]\u001b[A\n",
      " 77%|███████▋  | 382/493 [10:33<08:11,  4.43s/it]\u001b[A\n",
      " 78%|███████▊  | 383/493 [10:37<07:43,  4.22s/it]\u001b[A\n",
      " 78%|███████▊  | 384/493 [10:40<07:14,  3.98s/it]\u001b[A\n",
      " 78%|███████▊  | 385/493 [10:43<06:47,  3.78s/it]\u001b[A\n",
      " 78%|███████▊  | 386/493 [10:47<06:39,  3.74s/it]\u001b[A\n",
      " 78%|███████▊  | 387/493 [10:50<06:25,  3.63s/it]\u001b[A\n",
      " 79%|███████▊  | 388/493 [10:54<06:11,  3.54s/it]\u001b[A\n",
      " 79%|███████▉  | 389/493 [10:57<06:06,  3.53s/it]\u001b[A\n",
      " 79%|███████▉  | 390/493 [11:01<06:02,  3.52s/it]\u001b[A\n",
      " 79%|███████▉  | 391/493 [11:04<05:54,  3.48s/it]\u001b[A\n",
      " 80%|███████▉  | 392/493 [11:07<05:45,  3.42s/it]\u001b[A\n",
      " 80%|███████▉  | 393/493 [11:11<05:40,  3.41s/it]\u001b[A\n",
      " 80%|███████▉  | 394/493 [11:14<05:35,  3.39s/it]\u001b[A\n",
      " 80%|████████  | 395/493 [11:15<04:34,  2.81s/it]\u001b[A\n",
      " 80%|████████  | 396/493 [11:17<03:46,  2.34s/it]\u001b[A\n",
      " 81%|████████  | 397/493 [11:18<03:10,  1.98s/it]\u001b[A\n",
      " 81%|████████  | 398/493 [11:19<02:42,  1.71s/it]\u001b[A\n",
      " 81%|████████  | 399/493 [11:20<02:22,  1.52s/it]\u001b[A\n",
      " 81%|████████  | 400/493 [11:21<02:08,  1.38s/it]\u001b[A\n",
      " 81%|████████▏ | 401/493 [11:22<01:58,  1.29s/it]\u001b[A\n",
      " 82%|████████▏ | 402/493 [11:23<01:51,  1.23s/it]\u001b[A\n",
      " 82%|████████▏ | 403/493 [11:24<01:47,  1.19s/it]\u001b[A\n",
      " 82%|████████▏ | 404/493 [11:25<01:43,  1.16s/it]\u001b[A\n",
      " 82%|████████▏ | 405/493 [11:26<01:39,  1.13s/it]\u001b[A\n",
      " 82%|████████▏ | 406/493 [11:28<01:37,  1.12s/it]\u001b[A\n",
      " 83%|████████▎ | 407/493 [11:29<01:35,  1.11s/it]\u001b[A\n",
      " 83%|████████▎ | 408/493 [11:30<01:34,  1.11s/it]\u001b[A\n",
      " 83%|████████▎ | 409/493 [11:31<01:33,  1.11s/it]\u001b[A\n",
      " 83%|████████▎ | 410/493 [11:32<01:33,  1.12s/it]\u001b[A\n",
      " 83%|████████▎ | 411/493 [11:33<01:35,  1.16s/it]\u001b[A\n",
      " 84%|████████▎ | 412/493 [11:36<02:01,  1.50s/it]\u001b[A\n",
      " 84%|████████▍ | 413/493 [11:37<01:51,  1.39s/it]\u001b[A\n",
      " 84%|████████▍ | 414/493 [11:38<01:38,  1.25s/it]\u001b[A\n",
      " 84%|████████▍ | 415/493 [11:39<01:31,  1.17s/it]\u001b[A\n",
      " 84%|████████▍ | 416/493 [11:40<01:26,  1.12s/it]\u001b[A\n",
      " 85%|████████▍ | 417/493 [11:41<01:22,  1.09s/it]\u001b[A\n",
      " 85%|████████▍ | 418/493 [11:42<01:18,  1.05s/it]\u001b[A\n",
      " 85%|████████▍ | 419/493 [11:43<01:20,  1.09s/it]\u001b[A\n",
      " 85%|████████▌ | 420/493 [11:46<02:05,  1.72s/it]\u001b[A\n",
      " 85%|████████▌ | 421/493 [11:49<02:34,  2.15s/it]\u001b[A\n",
      " 86%|████████▌ | 422/493 [11:52<02:52,  2.43s/it]\u001b[A\n",
      " 86%|████████▌ | 423/493 [11:55<03:01,  2.59s/it]\u001b[A\n",
      " 86%|████████▌ | 424/493 [11:58<03:07,  2.72s/it]\u001b[A\n",
      " 86%|████████▌ | 425/493 [12:01<03:13,  2.85s/it]\u001b[A\n",
      " 86%|████████▋ | 426/493 [12:04<03:16,  2.94s/it]\u001b[A\n",
      " 87%|████████▋ | 427/493 [12:08<03:19,  3.02s/it]\u001b[A\n",
      " 87%|████████▋ | 428/493 [12:11<03:16,  3.03s/it]\u001b[A\n",
      " 87%|████████▋ | 429/493 [12:14<03:19,  3.12s/it]\u001b[A\n",
      " 87%|████████▋ | 430/493 [12:18<03:27,  3.29s/it]\u001b[A\n",
      " 87%|████████▋ | 431/493 [12:21<03:32,  3.42s/it]\u001b[A\n",
      " 88%|████████▊ | 432/493 [12:25<03:30,  3.45s/it]\u001b[A\n",
      " 88%|████████▊ | 433/493 [12:29<03:28,  3.48s/it]\u001b[A\n",
      " 88%|████████▊ | 434/493 [12:32<03:27,  3.52s/it]\u001b[A\n",
      " 88%|████████▊ | 435/493 [12:36<03:24,  3.52s/it]\u001b[A\n",
      " 88%|████████▊ | 436/493 [12:39<03:23,  3.58s/it]\u001b[A\n",
      " 89%|████████▊ | 437/493 [12:43<03:21,  3.59s/it]\u001b[A\n",
      " 89%|████████▉ | 438/493 [12:47<03:17,  3.58s/it]\u001b[A\n",
      " 89%|████████▉ | 439/493 [12:50<03:13,  3.58s/it]\u001b[A\n",
      " 89%|████████▉ | 440/493 [12:54<03:20,  3.78s/it]\u001b[A\n",
      " 89%|████████▉ | 441/493 [12:59<03:21,  3.88s/it]\u001b[A\n",
      " 90%|████████▉ | 442/493 [13:02<03:15,  3.84s/it]\u001b[A\n",
      " 90%|████████▉ | 443/493 [13:06<03:09,  3.80s/it]\u001b[A\n",
      " 90%|█████████ | 444/493 [13:10<03:07,  3.83s/it]\u001b[A\n",
      " 90%|█████████ | 445/493 [13:14<03:02,  3.80s/it]\u001b[A\n",
      " 90%|█████████ | 446/493 [13:17<02:55,  3.73s/it]\u001b[A\n",
      " 91%|█████████ | 447/493 [13:21<02:51,  3.73s/it]\u001b[A\n",
      " 91%|█████████ | 448/493 [13:25<02:47,  3.73s/it]\u001b[A\n",
      " 91%|█████████ | 449/493 [13:28<02:44,  3.74s/it]\u001b[A\n",
      " 91%|█████████▏| 450/493 [13:32<02:40,  3.73s/it]\u001b[A\n",
      " 91%|█████████▏| 451/493 [13:36<02:34,  3.69s/it]\u001b[A\n",
      " 92%|█████████▏| 452/493 [13:40<02:34,  3.76s/it]\u001b[A\n",
      " 92%|█████████▏| 453/493 [13:43<02:31,  3.79s/it]\u001b[A\n",
      " 92%|█████████▏| 454/493 [13:47<02:26,  3.77s/it]\u001b[A\n",
      " 92%|█████████▏| 455/493 [13:51<02:22,  3.75s/it]\u001b[A\n",
      " 92%|█████████▏| 456/493 [13:53<02:03,  3.33s/it]\u001b[A\n",
      " 93%|█████████▎| 457/493 [13:56<01:48,  3.02s/it]\u001b[A\n",
      " 93%|█████████▎| 458/493 [13:58<01:40,  2.88s/it]\u001b[A\n",
      " 93%|█████████▎| 459/493 [14:00<01:32,  2.73s/it]\u001b[A\n",
      " 93%|█████████▎| 460/493 [14:03<01:25,  2.61s/it]\u001b[A\n",
      " 94%|█████████▎| 461/493 [14:05<01:21,  2.53s/it]\u001b[A\n",
      " 94%|█████████▎| 462/493 [14:07<01:16,  2.46s/it]\u001b[A\n",
      " 94%|█████████▍| 463/493 [14:10<01:12,  2.41s/it]\u001b[A\n",
      " 94%|█████████▍| 464/493 [14:12<01:10,  2.42s/it]\u001b[A\n",
      " 94%|█████████▍| 465/493 [14:15<01:07,  2.41s/it]\u001b[A\n",
      " 95%|█████████▍| 466/493 [14:17<01:02,  2.32s/it]\u001b[A\n",
      " 95%|█████████▍| 467/493 [14:19<00:58,  2.26s/it]\u001b[A\n",
      " 95%|█████████▍| 468/493 [14:21<00:55,  2.22s/it]\u001b[A\n",
      " 95%|█████████▌| 469/493 [14:23<00:52,  2.18s/it]\u001b[A\n",
      " 95%|█████████▌| 470/493 [14:25<00:49,  2.15s/it]\u001b[A\n",
      " 96%|█████████▌| 471/493 [14:27<00:47,  2.15s/it]\u001b[A\n",
      " 96%|█████████▌| 472/493 [14:29<00:41,  1.98s/it]\u001b[A\n",
      " 96%|█████████▌| 473/493 [14:30<00:36,  1.82s/it]\u001b[A\n",
      " 96%|█████████▌| 474/493 [14:32<00:32,  1.71s/it]\u001b[A\n",
      " 96%|█████████▋| 475/493 [14:33<00:29,  1.65s/it]\u001b[A\n",
      " 97%|█████████▋| 476/493 [14:35<00:27,  1.60s/it]\u001b[A\n",
      " 97%|█████████▋| 477/493 [14:36<00:24,  1.53s/it]\u001b[A\n",
      " 97%|█████████▋| 478/493 [14:37<00:21,  1.46s/it]\u001b[A\n",
      " 97%|█████████▋| 479/493 [14:39<00:19,  1.42s/it]\u001b[A\n",
      " 97%|█████████▋| 480/493 [14:40<00:18,  1.40s/it]\u001b[A\n",
      " 98%|█████████▊| 481/493 [14:41<00:16,  1.37s/it]\u001b[A\n",
      " 98%|█████████▊| 482/493 [14:43<00:15,  1.37s/it]\u001b[A\n",
      " 98%|█████████▊| 483/493 [14:44<00:13,  1.39s/it]\u001b[A\n",
      " 98%|█████████▊| 484/493 [14:46<00:14,  1.59s/it]\u001b[A\n",
      " 98%|█████████▊| 485/493 [14:48<00:12,  1.60s/it]\u001b[A\n",
      " 99%|█████████▊| 486/493 [14:49<00:11,  1.59s/it]\u001b[A\n",
      " 99%|█████████▉| 487/493 [14:51<00:09,  1.60s/it]\u001b[A\n",
      " 99%|█████████▉| 488/493 [14:53<00:08,  1.61s/it]\u001b[A\n",
      " 99%|█████████▉| 489/493 [14:54<00:06,  1.62s/it]\u001b[A\n",
      " 99%|█████████▉| 490/493 [14:56<00:04,  1.63s/it]\u001b[A\n",
      "100%|█████████▉| 491/493 [14:57<00:03,  1.60s/it]\u001b[A\n",
      "100%|█████████▉| 492/493 [14:59<00:01,  1.61s/it]\u001b[A\n",
      "100%|██████████| 493/493 [15:01<00:00,  1.83s/it]\u001b[A\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train data size      (493, 14)\n",
      "===================\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 2.91938e+10\tvalid_0's mse_score: 2.91938e+10\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 4.05356e+10\tvalid_0's mse_score: 4.05356e+10\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 2.21098e+10\tvalid_0's mse_score: 2.21098e+10\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 2.81309e+10\tvalid_0's mse_score: 2.81309e+10\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 1.65064e+10\tvalid_0's mse_score: 1.65064e+10\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 2.30566e+10\tvalid_0's mse_score: 2.30566e+10\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 3.70353e+10\tvalid_0's mse_score: 3.70353e+10\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 4.6812e+10\tvalid_0's mse_score: 4.6812e+10\n",
      "Training until validation scores don't improve for 100 rounds\n",
      "[100]\tvalid_0's l2: 6.28107e+09\tvalid_0's mse_score: 6.28107e+09\n",
      "Early stopping, best iteration is:\n",
      "[1]\tvalid_0's l2: 9.68204e+09\tvalid_0's mse_score: 9.68204e+09\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|▍         | 1/22 [22:12<7:46:22, 1332.51s/it]\n",
      "0it [00:00, ?it/s]\u001b[A\n",
      "1it [00:05,  5.05s/it]\u001b[A\n",
      "2it [00:11,  5.37s/it]\u001b[A\n",
      "3it [00:17,  5.54s/it]\u001b[A\n",
      "4it [00:23,  5.92s/it]\u001b[A\n",
      "5it [00:30,  6.15s/it]\u001b[A\n",
      "6it [00:38,  6.68s/it]\u001b[A\n",
      "7it [00:45,  6.91s/it]\u001b[A\n",
      "8it [00:54,  7.29s/it]\u001b[A\n",
      "9it [01:02,  7.50s/it]\u001b[A\n",
      "10it [01:10,  7.70s/it]\u001b[A\n",
      "11it [01:28, 10.83s/it]\u001b[A\n",
      "12it [01:38, 10.54s/it]\u001b[A\n",
      "13it [01:48, 10.44s/it]\u001b[A\n",
      "14it [01:58, 10.36s/it]\u001b[A\n",
      "15it [02:09, 10.53s/it]\u001b[A\n",
      "16it [02:20, 10.59s/it]\u001b[A\n",
      "17it [02:30, 10.52s/it]\u001b[A\n",
      "18it [02:40, 10.38s/it]\u001b[A\n",
      "19it [02:50, 10.20s/it]\u001b[A\n",
      "20it [03:02, 10.60s/it]\u001b[A\n",
      "21it [03:14, 11.14s/it]\u001b[A\n",
      "22it [03:26, 11.38s/it]\u001b[A\n",
      "23it [03:39, 11.79s/it]\u001b[A\n",
      "24it [03:52, 12.14s/it]\u001b[A\n",
      "25it [04:17, 16.17s/it]\u001b[A\n",
      "26it [04:31, 15.43s/it]\u001b[A\n",
      "27it [04:45, 15.01s/it]\u001b[A\n",
      "28it [04:59, 14.77s/it]\u001b[A\n",
      "29it [05:15, 15.05s/it]\u001b[A\n",
      "30it [05:31, 15.51s/it]\u001b[A\n",
      "31it [05:47, 15.54s/it]\u001b[A\n",
      "32it [06:03, 15.58s/it]\u001b[A\n",
      "33it [06:33, 19.92s/it]\u001b[A\n",
      "34it [07:06, 23.82s/it]\u001b[A\n",
      "35it [07:22, 21.48s/it]\u001b[A\n",
      "36it [07:56, 25.37s/it]\u001b[A\n",
      "37it [08:23, 25.72s/it]\u001b[A\n",
      "38it [08:47, 25.28s/it]\u001b[A\n",
      "39it [09:17, 26.57s/it]\u001b[A\n",
      "40it [09:43, 26.53s/it]\u001b[A\n",
      "41it [10:22, 30.16s/it]\u001b[A\n",
      "42it [11:18, 37.85s/it]\u001b[A\n",
      "43it [12:49, 17.89s/it]\u001b[A\n",
      "  5%|▍         | 1/22 [35:02<12:15:45, 2102.18s/it]\n"
     ]
    },
    {
     "ename": "MemoryError",
     "evalue": "Unable to allocate array with shape (1, 38607058) and data type object",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-5-f0852caeaef3>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mroute\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtest_order_belong_to_trace\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m     \u001b[0mroute_order_info\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_train_order_by_route\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mroute\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m     \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'==================='\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m     \u001b[0mtrain_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mget_train_data\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mroute_order_info\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mroute\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m     \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'==================='\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-4-86dce72b31e8>\u001b[0m in \u001b[0;36mget_train_order_by_route\u001b[1;34m(route)\u001b[0m\n\u001b[0;32m      9\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mvalid_order_name\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msize\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     10\u001b[0m             \u001b[0mvalid_order_info\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mchunk\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mchunk\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0misin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalid_order_name\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 11\u001b[1;33m             \u001b[0mtrain_order_by_route\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconcat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mtrain_order_by_route\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mvalid_order_info\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     12\u001b[0m \u001b[1;31m#             break\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     13\u001b[0m     \u001b[0mtrain_order_by_route\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;34m'loadingOrder'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'carrierName'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'timestamp'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'longitude'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'latitude'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'vesselMMSI'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'speed'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'direction'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34m'TRANSPORT_TRACE'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Program\\Anaconda\\envs\\AI\\lib\\site-packages\\pandas\\core\\reshape\\concat.py\u001b[0m in \u001b[0;36mconcat\u001b[1;34m(objs, axis, join, join_axes, ignore_index, keys, levels, names, verify_integrity, sort, copy)\u001b[0m\n\u001b[0;32m    256\u001b[0m     )\n\u001b[0;32m    257\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 258\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mop\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_result\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    259\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    260\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Program\\Anaconda\\envs\\AI\\lib\\site-packages\\pandas\\core\\reshape\\concat.py\u001b[0m in \u001b[0;36mget_result\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    471\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    472\u001b[0m             new_data = concatenate_block_managers(\n\u001b[1;32m--> 473\u001b[1;33m                 \u001b[0mmgrs_indexers\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnew_axes\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mconcat_axis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    474\u001b[0m             )\n\u001b[0;32m    475\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Program\\Anaconda\\envs\\AI\\lib\\site-packages\\pandas\\core\\internals\\managers.py\u001b[0m in \u001b[0;36mconcatenate_block_managers\u001b[1;34m(mgrs_indexers, axes, concat_axis, copy)\u001b[0m\n\u001b[0;32m   2052\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2053\u001b[0m             b = make_block(\n\u001b[1;32m-> 2054\u001b[1;33m                 \u001b[0mconcatenate_join_units\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mjoin_units\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mconcat_axis\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2055\u001b[0m                 \u001b[0mplacement\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mplacement\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2056\u001b[0m             )\n",
      "\u001b[1;32mD:\\Program\\Anaconda\\envs\\AI\\lib\\site-packages\\pandas\\core\\internals\\concat.py\u001b[0m in \u001b[0;36mconcatenate_join_units\u001b[1;34m(join_units, concat_axis, copy)\u001b[0m\n\u001b[0;32m    251\u001b[0m     to_concat = [\n\u001b[0;32m    252\u001b[0m         \u001b[0mju\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_reindexed_values\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mempty_dtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mempty_dtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mupcasted_na\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mupcasted_na\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 253\u001b[1;33m         \u001b[1;32mfor\u001b[0m \u001b[0mju\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mjoin_units\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    254\u001b[0m     ]\n\u001b[0;32m    255\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Program\\Anaconda\\envs\\AI\\lib\\site-packages\\pandas\\core\\internals\\concat.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m    251\u001b[0m     to_concat = [\n\u001b[0;32m    252\u001b[0m         \u001b[0mju\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_reindexed_values\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mempty_dtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mempty_dtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mupcasted_na\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mupcasted_na\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 253\u001b[1;33m         \u001b[1;32mfor\u001b[0m \u001b[0mju\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mjoin_units\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    254\u001b[0m     ]\n\u001b[0;32m    255\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Program\\Anaconda\\envs\\AI\\lib\\site-packages\\pandas\\core\\internals\\concat.py\u001b[0m in \u001b[0;36mget_reindexed_values\u001b[1;34m(self, empty_dtype, upcasted_na)\u001b[0m\n\u001b[0;32m    234\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    235\u001b[0m             \u001b[1;32mfor\u001b[0m \u001b[0max\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mindexer\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mindexers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 236\u001b[1;33m                 \u001b[0mvalues\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0malgos\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtake_nd\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mindexer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0max\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfill_value\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mfill_value\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    237\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    238\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mvalues\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\Program\\Anaconda\\envs\\AI\\lib\\site-packages\\pandas\\core\\algorithms.py\u001b[0m in \u001b[0;36mtake_nd\u001b[1;34m(arr, indexer, axis, out, fill_value, mask_info, allow_fill)\u001b[0m\n\u001b[0;32m   1714\u001b[0m             \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mempty\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mout_shape\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0morder\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"F\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1715\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1716\u001b[1;33m             \u001b[0mout\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mempty\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mout_shape\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1717\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1718\u001b[0m     func = _get_take_nd_function(\n",
      "\u001b[1;31mMemoryError\u001b[0m: Unable to allocate array with shape (1, 38607058) and data type object"
     ]
    }
   ],
   "source": [
    "for route in tqdm(test_order_belong_to_trace):\n",
    "    route_order_info = get_train_order_by_route(route)\n",
    "    print('===================')\n",
    "    train_data = get_train_data(route_order_info, route)\n",
    "    print('===================')\n",
    "    \n",
    "    features = [c for c in train_data.columns if c not in ['loadingOrder', 'label', 'carrierName', 'vesselMMSI', 'direction', 'TRANSPORT_TRACE']]\n",
    "    model_by_route = train_model(train_data[features], train_data['label'])\n",
    "    \n",
    "    for order in test_order_belong_to_trace[route]:\n",
    "        test_order_data = get_test_data(order)\n",
    "#         print(test_order_data)\n",
    "        res = model_by_route.predict(test_order_data[features], num_iteration=model_by_route.best_iteration)\n",
    "        test_data_origin.loc[test_data_origin['loadingOrder'] == order, 'ETA'] = (test_data_origin[test_data_origin['loadingOrder'] == order]['onboardDate'] + pd.Timedelta(seconds=res[0])).apply(lambda x:x.strftime('%Y/%m/%d  %H:%M:%S'))\n",
    "#         print(test_data_origin[test_data_origin['loadingOrder'] == order])\n",
    "#         break\n",
    "#     break\n",
    "    route_order_info = None\n",
    "    train_data = None\n",
    "\n",
    "test_data_origin['creatDate'] = pd.datetime.now().strftime('%Y/%m/%d  %H:%M:%S')\n",
    "test_data_origin['timestamp'] = test_data_origin['temp_timestamp']\n",
    "\n",
    "result = test_data_origin[['loadingOrder', 'timestamp', 'longitude', 'latitude', 'carrierName', 'vesselMMSI', 'onboardDate', 'ETA', 'creatDate']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "result.to_csv(result_path)\n",
    "# mox.file.copy_parallel(result_path, OBS_RES_PATH)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.6 64-bit ('AI': conda)",
   "language": "python",
   "name": "python37664bitaiconda6859e03b37c34f0182c9bde8073269f7"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
